http = require 'http'
https = require 'https'
T = require './lib/tools.coffee'
url = require 'url'
iconv = require 'iconv-lite'
qs = require 'querystring'
cheerio = require 'cheerio'
events = (require 'events').EventEmitter
fs = require 'fs'
SeqQueue = require 'seq-queue'
urlMap = {
    jd:'http://dd.search.jd.com/',
    yhd:'http://www.yhd.com/ctg/get_new_keywords.do',
    tb:'http://suggest.taobao.com/sug',
    yixun:'http://ds.yixun.com/cgi-bin/isuggest_yixun',
    etao:'http://suggest.etao.com/sug',
    amazon:'http://completion.amazon.cn/search/complete',
    baidu:'http://suggestion.baidu.com/su',
    dd:'http://schprompt.dangdang.com/suggest_new.php',
    bing:'http://bj1.api.bing.com/qsonhs.aspx',
    360:'http://sug.so.360.cn/suggest/word',
    soso:'http://w.sugg.sogou.com/sugg/ajaj_json.jsp',
    google:'https://www.google.com.hk/complete/search'
}

class Request extends events
    constructor:()->
        @u2 = null
        # @urlQueue = [] 
    request:(urlS,callback)=>
        # console.log urlString
        # if @urlQueue.length is 0 then return
            # urlS = @urlQueue.shift()
        if typeof urlS is 'string'
            urlObj = url.parse(urlS)
        else
            urlObj = urlS
        # console.log urlObj.protocol
        if urlObj.protocol == 'http:'
            protocolReq = http.request
            urlObj.port = 80
        else if urlObj.protocol == 'https:'
            protocolReq = https.request
            urlObj.port = 443
        # console.log protocolReq,urlObj
        # console.log urlS,urlObj
        urlObj.method = 'GET'
        if !callback
            callbck = ()->
                return
        T.log url.format urlS
        req = protocolReq(urlObj, (res)=> 
            # @emit 'onFetchstart', url.parse(urlS),req
            # console.log("Got response: " + res.statusCode, res.headers)
            if(res.statusCode is 301 or res.statusCode is 302)
                @u2= res.headers.location
                if (@u2.indexOf 'http://') isnt 0 and (@u2.indexOf 'https://') isnt 0
                    @u2 = urlObj.protocol+'//'+ urlObj.host + decodeURIComponent(res.headers.location)
                console.log urlS + '  forword to  '+ @u2
                # @emit 'onFetchredirect' ,url.parse(urlS),@u2,res
                @request @u2,callback
            buffers = []
            size = 0
            # console.log c++
            res
            .on('data', (buffer)=> 
                buffers.push(buffer)
                size += buffer.length
            )
            .on('end', ()=> 
                T.logEnd url.format urlS
                responseBuffer = new Buffer size
                pos = 0;
                try
                    if buffers.length isnt 0
                        # console.log (res.headers['content-type'].match /charset=(\w|\s|-|\d)*/)[0].toLowerCase()
                        @charset =  (res.headers['content-type'].match /charset=(\w|\s|-|\d)*/)[0].toLowerCase()
                        @charset = @charset.split('=')[1]
                        # console.log 123213213213, @charset
                        for i in [0..buffers.length-1]
                            buffers[i].copy responseBuffer, pos
                            pos += buffers[i].length
                    # T.logEnd urlS
                        if typeof urlS is 'string'
                            callback url.parse(urlS),responseBuffer,res 
                        else
                            callback urlS,responseBuffer,res
                catch e
                    console.log e
                    # errStr = JSON.stringify e
                    # throw new Error()

                # @emit 'onFetchComplete',url.parse(urlS),responseBuffer,res
            )

        )
        req.on('error', (e)=>
            # @emit 'onFetchclienterror',e
            console.log("Got error: " + e.message)
            # @request urlS,callback
            console.log urlS
            # throw e
            # request urlS,callback
        )
        # req.write('')
        req.end()

class GetSearchItem extends events
    constructor:()->
        # @funList = [@getDangdangSearch,@getYHDSearch,@getYiXunSearch,@getJDSearch,@getBaiduSearch,@getAmazonSearch,@getTaoBaoSearch,@geteTaoSearch,@getBingSearch,@get360Search,@getSosoSearch]
        # @funList = [@getDangdangSearch,@getYiXunSearch,@getJDSearch,@getBaiduSearch,@getAmazonSearch,@getTaoBaoSearch,@geteTaoSearch,@getBingSearch,@get360Search,@getSosoSearch]
        @req = new Request();
        # @siteArr = ['一淘','亚马逊','京东','当当','淘宝','百度','易讯','1号店','Bing','360','soso']
        @siteArr = [{'一淘':@geteTaoSearch},{'亚马逊':@getAmazonSearch},{'京东':@getJDSearch},{'当当':@getDangdangSearch},{'淘宝':@getTaoBaoSearch},{'百度':@getBaiduSearch},{'易讯':@getYiXunSearch},{'Bing':@getBingSearch},{'360':@get360Search},{'soso':@getSosoSearch},{'1号店':@getYHDSearch},{'Google':@getGoogleSearch}]
    getSosoSearch:(keyword,callback)=>
        param = { 
            key: keyword
        }
        urlStr = urlMap.soso+'?'+qs.stringify(param)
        @req.request urlStr,(urlObj,responseBuffer,res)=>
            window = {}
            window.sogou = {}
            window.sogou.sug = (arg)->
                return arg[1]
            try
                # str = iconv.decode responseBuffer,'GBK'
                str = iconv.decode responseBuffer,@req.charset
                obj = eval str
                arr = []
                for i in obj
                    arr.push(i)
            catch e
                arr = []
            @emit 'data' , 'soso',keyword,arr
            if callback then callback 'soso',keyword,arr        
    get360Search:(keyword,callback)=>
        pstr = 'callback=suggest_so&encodein=utf-8&encodeout=utf-8&word=1'
        param = { 
            callback: 'cb',
            encodein: 'utf-8',
            encodeout: 'utf-8',
            word: keyword 
        }
        # urlStr = urlMap.bing+'?'+qs.stringify(param)+'&o=a+ds+ds+p' 
        urlStr = urlMap[360]+'?'+qs.stringify(param)
        @req.request urlStr,(urlObj,responseBuffer,res)=>
            cb = (arg)->
                return arg
            try
                str = iconv.decode responseBuffer,@req.charset
                obj = (eval str).s
                # console.log obj
                arr = []
                for i in obj
                    arr.push(i)
            catch e
                arr = []
            @emit 'data' , '360',keyword,arr
            if callback then callback '360',keyword,arr

    getBingSearch:(keyword,callback)=>
        pstr = 'FORM=ASAPIH&mkt=zh-CN&type=cb&cb=sa_inst.apiCB&cp=0&q=&o=a+ds+ds+p'
        param = { 
            # FORM: 'ASAPIH',
            mkt: 'zh-CN',
            # type: 'cb',
            cb: 'cb',
            # cp: '0',
            q: keyword
            
        }
        urlStr = urlMap.bing+'?'+qs.stringify(param)+'&o=a+ds+ds+p' 
        @req.request urlStr,(urlObj,responseBuffer,res)=>
            try
                str = iconv.decode responseBuffer,@req.charset
                # console.log `eval(str)`
                rst = JSON.parse(str)
                obj = rst.AS.Results[0].Suggests
                # console.log _str
                arr = []
                for i in obj
                    arr.push(i.Txt)
            catch e
                arr = []
            @emit 'data' , 'Bing',keyword,arr
            if callback then callback 'Bing',keyword,arr    

    getDangdangSearch:(keyword,callback)=>
        query = 'keyword='+T.encodeURIComponent_GBK keyword
        urlStr = urlMap.dd+'?'+query
        # urlStr = 'http://schprompt.dangdang.com/suggest_new.php?keyword=%D0%AC%D7%D3&catalog=&guanid=&0.9113395449239761'
        # console.log query
        @req.request urlStr,(urlObj,responseBuffer,res)=>
            try
                str = iconv.decode responseBuffer,@req.charset
                # console.log `eval(str)`
                _str = (str.split ';')[0]
                # console.log _str
                eval _str
                arr = []
                # console.log json_data
                obj = json_data[2]
                for i in obj
                    for k of i
                        arr.push(k)
                # console.log arr
            catch e
                arr = []
            @emit 'data' , '当当',keyword,arr
            if callback then callback '当当',keyword,arr
    getTaoBaoSearch : (keyword,callback)=>
        param = {
            q:keyword,
            code:'utf-8'
        }
        query = qs.stringify(param)
        urlStr = urlMap.tb+'?'+ query
        # urlStr = 'http://suggest.taobao.com/sug?code=utf-8&q=%E6%9B%BC%E5%BE%B7'
        @req.request urlStr,(urlObj,responseBuffer,res)=>
            try
                str = iconv.decode responseBuffer,@req.charset
                obj = JSON.parse(str)
                # console.log obj.result
                arr = []
                for i in obj.result
                    arr.push(i[0] )
            catch e
                arr = []
            @emit 'data' , '淘宝',keyword,arr
            if callback then callback '淘宝',keyword,arr

    getJDSearch:(keyword,callback)=>
        param = {
            key:keyword
        }
        query = qs.stringify(param)
        urlStr = urlMap.jd+'?'+ query
        @req.request urlStr,(urlObj,responseBuffer,res)=>
            try
                str = iconv.decode responseBuffer,@req.charset
                obj = JSON.parse(str)
                arr = []
                for i in obj
                    arr.push(i.keyword)
            catch e
                arr = []
            @emit 'data' , '京东',keyword,arr
            if callback then callback '京东',keyword,arr

    getYHDSearch:(keyword,callback)=>
        param = {
            keyword:keyword
        }
        query = qs.stringify(param)
        urlStr = urlMap.yhd+'?'+ query
        # urlStr = 'http://www.yhd.com/ctg/get_new_keywords.do?keyword=s'
        # console.log urlStr
        @req.request urlStr,(urlObj,responseBuffer,res)=>
            try
                str = iconv.decode responseBuffer,@req.charset
                # console.log str
                # console.log str
                # console.log str
                obj = JSON.parse(str)
                $ = cheerio.load(obj.value)
                spanLsit = $('span')
                arr = []
                for span in spanLsit
                    arr.push($(span).text())
            catch e
                arr = []
                # console.log $(span).text()
            @emit 'data' , '1号店',keyword,arr
            if callback then callback '1号店',keyword,arr

    getYiXunSearch:(keyword,callback)=>
        param = {
            KeyWord:keyword,
            AreaCode:1
        }
        options = url.parse( urlMap.yixun+'?'+qs.stringify param)
        options.headers = {
            'Referer':'http://www.yixun.com/'
        }
        @req.request options,(urlObj,responseBuffer,res)=>
            try
                str = iconv.decode responseBuffer,@req.charset
                BBC = {}
                BBC.head = {}
                BBC.head.smartBoxCallback = (p)->
                    return p
                obj = eval(str)
                arr = []

                obj.list?=[]
                for i in obj.list
                    arr.push(i[0])
            catch e
                arr = []
            @emit 'data' , '易讯',keyword,arr
            if callback then callback '易讯',keyword,arr
            # console.log 
    geteTaoSearch : (keyword,callback)=>
        param = {
            q:keyword,
            code:'utf-8',
            area:'etao'
        }
        query = qs.stringify(param)
        urlStr = urlMap.etao+'?'+ query
        @req.request urlStr,(urlObj,responseBuffer,res)=>
            try
                str = iconv.decode responseBuffer,@req.charset
                obj = JSON.parse(str).result
                arr = []
                for i in obj
                    arr.push(i[0])
            catch e
                arr = []
                # console.log $(span).text()
            @emit 'data' , '一淘',keyword,arr
            if callback then callback '一淘',keyword,arr
    getBaiduSearch : (keyword,callback)=>
        param = { 
            wd: keyword,
            cb: 'cb',
        }
        query = qs.stringify param
        urlStr = urlMap.baidu+'?'+ query
        @req.request urlStr,(urlObj,responseBuffer,res)=>
            try
                str = iconv.decode responseBuffer,@req.charset
                cb = (arg)->
                    return arg
                obj = eval(str)
                arr = obj.s
            catch e
                arr = []
            @emit 'data' , '百度',keyword,arr
            if callback then callback '百度',keyword,arr
    getAmazonSearch : (keyword,callback)=>
        param = { 
            q: keyword,
            'search-alias': 'aps',
            mkt: '3240',
        }
        urlStr = urlMap.amazon+'?'+qs.stringify param
        @req.request urlStr,(urlObj,responseBuffer,res)=>
            try
                str = iconv.decode responseBuffer,@req.charset
                obj =  JSON.parse str
                arr = obj[1]
            catch e
                arr = []
            @emit 'data' , '亚马逊',keyword,arr
            if callback then callback '亚马逊',keyword,arr    
    getGoogleSearch : (keyword,callback)=>
        param = { 
            client: 'serp',
            hl: 'zh-CN',
            q: keyword,
            xhr: 't' 
        }
        urlStr = urlMap.google+'?'+qs.stringify param
        # urlStr = 'https://www.google.com.hk/complete/search?client=serp&hl=zh-CN&gs_rn=32&gs_ri=serp&pq=%E4%BB%80%E4%B9%88%E6%98%AF%20Security%20cookie&cp=2&gs_id=1jwn&q=13&xhr=t'
        urlStr = url.parse urlStr
        cookie = { 
            PREF: 'ID=fce3184a5a282b11:U=e4f6768da881abe4:FF=1:LD=zh-CN:NW=1:TM=1381759448:LM=1385619184:S=vLFrMF_NpxoIO5tj',
            NID: '67=hggSPPHG0mS4dS_GTUTB5uwioKiO2w69opCb6c-VsFupTjQWhuqznOrrVfdw-b4iOkGTcUNSAjzzDGjhKUUM1SfW25FlI0f_v4RZSHoEC75AzN7MK_slWWOpj44myoPi' 
        }
        # console.log qs.parse('client=serp&hl=zh-CN&gs_rn=32&gs_ri=serp&pq=%E4%BB%80%E4%B9%88%E6%98%AF%20Security%20cookie&cp=2&gs_id=1jwn&q=13&xhr=t')
        urlStr.headers = {
            cookie:T.cookieFormat cookie
        }
        
        # console.log urlStr
        @req.request urlStr,(urlObj,responseBuffer,res)=>
            arr = []
            try
                str = iconv.decode responseBuffer,@req.charset
                obj =  JSON.parse str
                for i in obj[1]
                    arr.push i[0]
                    # console.log i
            catch e
                arr = []
            @emit 'data' , 'Google',keyword,arr
            if callback then callback 'Google',keyword,arr    




# funArr = [getYHDSearch,getJDSearch,getTaoBaoSearch,geteTaoSearch,getDangdangSearch,getAmazonSearch,getBaiduSearch,getYiXunSearch]
clearFile = ()->
    dataPath = './data/keywords'
    gsl1 = new GetSearchItem()
    for item in gsl1.siteArr
        for i of item
            _dataPath = dataPath+'_'+i+'.csv'
        # console.log _dataPath
            delFile if (fs.unlinkSync _dataPath if fs.existsSync _dataPath)
KeywordMap = {}
discArr = (arr)->
    map = {}
    rstArr = []
    for i in arr
        map[i]=''
    for k of map
        rstArr.push k
    return rstArr
fun = (keyword,callback)->
    dataPath = './data/keywords'
    queue = SeqQueue.createQueue(1000)
    gsl = new GetSearchItem()
    pos = 0
    rstMap = {}
    rstArr = []

    iter = ()->
        item = gsl.siteArr[pos]
        for k of item
            fn = item[k]
        if fn 
            queue.push (task)->
                fn keyword.trim(),(wsite,kwd,sterms)->
                    # console.log wsite,kwd,sterms
                    task.done()
                    rstMap[wsite] ?= []
                    for i in sterms
                        rstMap[wsite].push i
                    # console.log wsite
                    # 回调后再次请求 关键词 + 空格
                    queue.push (task)->
                        fn keyword.trim()+' ',(wsite,kwd,sterms)->
                            # console.log wsite,kwd,sterms
                            task.done()
                            for i in sterms
                                rstMap[wsite].push i
                            # console.log wsite
                            # 去重
                            rstMap[wsite] = discArr rstMap[wsite]
                            if pos < gsl.siteArr.length
                                pos++
                                iter()


        else
            # for i of rstMap
            #     rstArr.push i
            if callback then callback keyword,rstMap
    iter()




# clearFile()

# arrList = ['阿克苏苹果','黄心猕猴桃','云南蒙自石榴','琯溪红心蜜柚','沾化冬枣']
# str = '苹果（APPLE）iPhone 5S 16G版 3G手机（深空灰色）WCDMA/GSM 诺基亚（NOKIA）Lumia 625 3G手机（橙色）三星 I8750 3G手机（灰色）'
# arrList = str.split(' ')
arrList = ['鞋子']
# arrList = ['移动电源','飞毛腿移动电源','电小二移动电源','移动电源排行','酷比魔方','音响','akg']
run = (arr)->
    dataPath = './data/keywords.csv'
    delFile if (fs.unlinkSync dataPath if fs.existsSync dataPath)
    q = SeqQueue.createQueue(1000)
    pos = 0
    iter = ()->
        if pos < arr.length
            q.push (task)->
                fun arr[pos],(keyword,rstMap)->
                    task.done()
                    # console.log rstMap
                    for k of rstMap
                        str = keyword+','+k+','+rstMap[k].join(',')+'\n'
                        fs.appendFileSync dataPath,str
                    # fs.appendFileSync dataPath ,([keyword,list.join(',')].join(',')+'\n')
                    # fs.appendFileSync dataPath, list.join(',') +'\n'
                    pos++
                    iter()
    iter()


run (arrList)
# fun '阿克苏苹果',(arr)->
#     console.log arr.length,arr




# console.log new Date().getTime()
test = (arg)->
    rq = new GetSearchItem()
    keyword = '鞋子'
    switch arg
        when 'yhd'
            console.log rq.getYHDSearch
            rq.getYHDSearch keyword,(wsite,keyword,arr)->
                console.log '1号店',keyword,arr        
        when 'jd'
            rq.getJDSearch keyword,(wsite,keyword,arr)->
                console.log '京东',keyword,arr
        when 'tb'
            rq.getTaoBaoSearch keyword,(wsite,keyword,arr)->
                console.log '淘宝',keyword,arr
        when 'yx'
            rq.getYiXunSearch keyword,(wsite,keyword,arr)->
                console.log '易讯',keyword,arr
        when '1tao'
            rq.geteTaoSearch keyword,(wsite,keyword,arr)->
                console.log '一淘',keyword,arr
        when 'baidu'
            rq.getBaiduSearch keyword,(wsite,keyword,arr)->
                console.log '百度',keyword,arr
        when 'amazon'
            rq.getAmazonSearch keyword,(wsite,keyword,arr)->
                console.log '亚马逊',keyword,arr
        when 'dangdang'
            rq.getDangdangSearch keyword,(wsite,keyword,arr)->
                console.log '当当',keyword,arr
        when 'bing'
            rq.getBingSearch keyword,(wsite,keyword,arr)->
                console.log 'Bing',keyword,arr
        when '360'
            rq.get360Search keyword,(wsite,keyword,arr)->
                console.log '360',keyword,arr
        when 'soso'
            rq.getSosoSearch keyword,(wsite,keyword,arr)->
                console.log 'soso',keyword,arr
        when 'google'
            rq.getGoogleSearch keyword,(wsite,keyword,arr)->
                console.log 'google',keyword,arr

# test 'jd'
# test 'yhd'
# test 'tb'
# test 'yx'
# test '1tao'
# test 'amazon'
# test 'dangdang' 
# test 'baidu'
# test 'bing'
# test '360'
# test 'soso'
# test 'google'